﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;


namespace XCrawler
{
    public class HttpCrawler : ICrawler, IDisposable
    {

        /// <summary>
        /// 构造
        /// </summary>
        public HttpCrawler() { }

        /// <summary>
        /// 构造
        /// </summary>
        /// <param name="crawlerUrl"></param>
        /// <param name="proxyAddress"></param>
        /// <param name="method"></param>
        /// <param name="userAgent"></param>
        /// <param name="timeout"></param>
        /// <param name="cookiesContainer"></param>
        public HttpCrawler(string crawlerUrl, string proxyAddress = "", string method = "GET", string userAgent = "", int timeout = 5000, string cookies="")
        {
            this.CrawlerUrl = new Uri(crawlerUrl);
            this.ProxyAddress = proxyAddress;
            this.Cookies = cookies;
            this.Method = method;
            this.Timeout = timeout;

            if (!string.IsNullOrWhiteSpace(userAgent))
            {
                this.UserAgent = userAgent;
            }

        }


        /// <summary>
        /// 爬虫启动事件
        /// </summary>
        public event EventHandler<OnStartEventArgs> OnStartEvent;
        /// <summary>
        /// 爬虫完成事件
        /// </summary>
        public event EventHandler<OnCompletedEventArgs> OnCompletedEvent;
        /// <summary>
        /// 爬虫出错事件
        /// </summary>
        public event EventHandler<OnErrorEventArgs> OnErrorEvent;
        /// <summary>
        /// 定义Cookie容器
        /// </summary>
        public string Cookies { get; set; }

        /// <summary>
        /// 爬行地址
        /// </summary>
        public Uri CrawlerUrl  { get; set; }

        /// <summary>
        /// 超时时间（毫秒）
        /// </summary>
        public int Timeout { get; set; } = 5*1000;

        /// <summary>
        /// 代理地址127.0.0.1:1080
        /// </summary>
        public string ProxyAddress { get; set; }


        /// <summary>
        /// 爬虫的请求方式
        /// </summary>
        public string Method { get; set; } = "GET";

        /// <summary>
        /// 用户请求模拟
        /// </summary>
        public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";


        /// <summary>
        /// 返回值解析器集合
        /// </summary>
        public IEnumerable<ICrawlerParser> Parsers { get; set; }



        protected void SetRequestProperties(HttpClientHandler handler, HttpRequestMessage request, ServicePoint servicePoint)
        {
            servicePoint.Expect100Continue = false;//加快载入速度
            servicePoint.UseNagleAlgorithm = false;//禁止Nagle算法加快载入速度
            servicePoint.ConnectionLimit = int.MaxValue;//定义最大连接数
            //允许的安全协议(全部加上)
            //ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;

            request.Headers.UserAgent.ParseAdd(this.UserAgent); //设置User-Agent，伪装成Google Chrome浏览器
            request.Headers.Accept.ParseAdd("*/*");
            request.Headers.Connection.ParseAdd("Keep-Alive");//启用长连接
            request.Headers.AcceptEncoding.ParseAdd("gzip,deflate");//定义gzip压缩页面支持
            request.Headers.Add(HttpRequestHeader.ContentType.ToString(), "application/x-www-form-urlencoded");
            request.Properties["RequestTimeout"] = this.Timeout;//定义请求超时时间为5秒

            handler.AllowAutoRedirect = false;//禁止自动跳转
 
            //设置代理服务器IP，伪装请求地址                                                    
            if (!string.IsNullOrWhiteSpace(this.ProxyAddress))
            {
                handler.Proxy = new WebProxy(this.ProxyAddress);
            }

            if (!string.IsNullOrWhiteSpace(this.Cookies))
            {
                handler.CookieContainer = this.Cookies.ToCookieContainer();
            }
        }

        /// <summary>
        /// 设置解析绑定的值
        /// </summary>
        /// <param name="parserBag"></param>
        /// <param name="request"></param>
        /// <param name="httpResponse"></param>
        protected void SetParserBagProperties(ParserBag parserBag, HttpRequestMessage request, HttpResponseMessage response = null)
        {
            if(response == null)
            {
                //开始计时
                parserBag.StartWatchTimer();
                parserBag.RequestHeaders = request.Headers;
                parserBag.CrawlerUrl = request.RequestUri;
            }
            else
            {
                parserBag.StatusCode = response.StatusCode;
                parserBag.ResponseHeaders = response.Content.Headers;
                parserBag.ContentType = response.Content.Headers.ContentType.MediaType;
                parserBag.ContentEncoding = response.Content.Headers.ContentEncoding.ToList().ListToString();
                parserBag.ContentCharset = response.Content.Headers.ContentType.CharSet;
            }


            

        }

            


        /// <summary>
        /// 开始爬行
        /// </summary>
        /// <returns></returns>
        public async Task StartAsync()
        {
            var parserBag = new ParserBag();
       

            var clientHandler = new HttpClientHandler();
            var client = new HttpClient(clientHandler);
            var request = new HttpRequestMessage(new HttpMethod(this.Method), this.CrawlerUrl);
            var servicePoint = ServicePointManager.FindServicePoint(CrawlerUrl);
            this.SetRequestProperties(clientHandler, request, servicePoint);
            this.SetParserBagProperties(parserBag, request, null);

            Exception unhandledException = null;
            HttpResponseMessage httpResponse = null;
            try
            {
                //开始回调
                this.OnStartEvent?.Invoke(this, new OnStartEventArgs(this.CrawlerUrl));

                //请求开始
                httpResponse = await client.SendAsync(request).WithTimeout(TimeSpan.FromMilliseconds(this.Timeout)).ConfigureAwait(false);
                var response = httpResponse.Content;
                var contentLength = response.Headers.ContentLength ?? throw new InvalidOperationException("Content length not specified");
               

                //将Cookie加入容器，保存登录状态
                if (httpResponse.Headers.Contains("Set-Cookie"))
                {
                    this.Cookies = httpResponse.Headers.GetValues("Set-Cookie").FirstOrDefault() ?? this.Cookies;
                }

                this.SetParserBagProperties(parserBag, request, httpResponse);


                //保存页面流
                parserBag.ResponseBuffer = new MemoryStream((int)contentLength);
                parserBag.ResponseBuffer = await response.ReadAsStreamAsync().ConfigureAwait(false); ;
 


                //循环处理所有解析器
                await this.Parsers.ForEach(crawlerParser => this.Execute(crawlerParser, parserBag)).ConfigureAwait(false);
                //停止计时
                parserBag.StopWatchTimer();
                //爬虫完成事件的回调
                this.OnCompletedEvent?.Invoke(this, new OnCompletedEventArgs(this.CrawlerUrl, parserBag));

                // Read the response into a Stream object. 
                //var responseStream = await response.ReadAsStreamAsync().ConfigureAwait(false);



                //await responseStream.CopyToAsync(parserBag.ResponseBuffer,
                //    async (source, dest, exception) =>
                //    {
                //        if (exception.IsNull())
                //        {
                //            //循环处理所有解析器
                //            await this.Parsers.ForEach(crawlerParser => this.Execute(crawlerParser, parserBag)).ConfigureAwait(false);
                //            //停止计时
                //            parserBag.WatchTimer.Stop();
                //            //爬虫完成事件的回调
                //            this.OnCompletedEvent?.Invoke(this, new OnCompletedEventArgs(this.CrawlerUrl, parserBag));
                //            //CallComplete(requestState, httpResponse);
                //        }
                //        else
                //        {
                //                        // Put delay here in case of error.
                //                        //DownloadWithRetryAsync(requestState, exception);
                //        }
                //    },
                //    bd =>
                //    {
                //        //正在进行中的回调

                //        //requestState.DownloadProgress?.Invoke(new DownloadProgressEventArgs
                //        //{
                //        //    Referrer = requestState.Referrer,
                //        //    Step = requestState.CrawlStep,
                //        //    BytesReceived = bd,
                //        //    TotalBytesToReceive = (uint)httpResponse.Content.Headers.ContentLength,
                //        //    DownloadTime = requestState.StartTime - DateTime.UtcNow,
                //        //});
                //    },
                //    32*1024,1024*1024,TimeSpan.FromSeconds(30)).ConfigureAwait(false);

            }
            catch (Exception ex)
            {
                //停止计时
                parserBag.StopWatchTimer();
                this.OnErrorEvent?.Invoke(this, new OnErrorEventArgs(this.CrawlerUrl, parserBag, ex));
            }

        }



        private async Task Execute(ICrawlerParser crawlerParser, ParserBag parserBag)
        {
            try
            {
                var sw = Stopwatch.StartNew();
                //this.m_Logger.Debug("Executing parser work {0}", crawlerParser.GetType().Name);
                if (crawlerParser is ICrawlerParser)
                {
                    await crawlerParser.DoWorkAsync(this, parserBag).ConfigureAwait(false);
                }
                //this.m_Logger.Debug("Executed parser work {0} in {1}", crawlerParser.GetType().Name, sw.Elapsed);
            }
            catch (Exception ex)
            {
                //this.OnProcessorException(propertyBag, ex);
            }
        }




        public void Dispose()
        {
             
        }
    }


}
